#!/usr/bin/env python

"""
Program that calls wget with specific options and outputs a file with a list of URLs. For example,
$ wget -r -l 1 -nd -nv -o output.txt --reject .css,.gif,.jpg,.js,.txt --spider http://illinois.edu/

After the wget output file has been produced, it calls the filter_urls function in filter_urls.py,
and then prints the filtered list of URLs that it returns.
"""

import optparse
import subprocess
import sys

from utils import StartingURL
from filter_urls import filter_urls

usage = "usage: %prog [options]"
parser = optparse.OptionParser(usage=usage)

def call_wget(options, url):
    reject_list = '.css,.gif,.jpg,.js,.txt';

    # construct command
    wget = ['wget']
    wget.extend(['-r', '-l', options.depth])

    if options.span == True:
        wget.extend(['--span-hosts', '--domains=%s' % url.next_level_domain])

    wget.extend(['-nd', '-nv', '-o', options.outfile])
    wget.extend(['--reject', reject_list])
    wget.extend(['--spider', options.url])

    print "Calling: %s" % ' '.join(wget)
    subprocess.call(wget)

def main():
    parser.add_option("-d", "--depth", dest="depth",
                      help="wget recursion depth: (must be 1 or 2)", metavar="<int>")

    parser.add_option("-s", "--span-hosts", dest="span", action="store_true", default="False",
                      help="span hosts (next level subdomains)")

    parser.add_option("-o", "--outfile", dest="outfile",
                      help="wget output file (list of URLs)", metavar="<file>")

    parser.add_option("-u", "--url", dest="url",
                      help="wget starting URL", metavar="<url>")

    (options, args) = parser.parse_args()

    # check options and args
    if (options.depth != '1' and options.depth != '2'):
        parser.print_help()
        sys.exit()

    starting_url = StartingURL(options.url)
    call_wget(options, starting_url)

    urls = filter_urls(options.outfile, starting_url, options.span)

    for url in urls:
        print url[0], url[1]

    print 'length: ', len(urls)

if __name__ == "__main__":
    main()
